package com.dmcb.trade.business.crawlers.author;

import com.dmcb.common.business.services.BaseService;
import com.dmcb.common.web.conversion.JsonResult;
import com.dmcb.common.business.utils.WebClientUtil;
import com.dmcb.trade.controller.ArticleCrawlerController;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

/**
 * 乐居网站文章导入
 * Created by wangyong on 2017/7/11.
 */
@Service
public class LeJuCrawler extends BaseService {

    @Autowired
    private ArticleCrawlerController articleCrawlerController;
    @Autowired
    private WebClientUtil webClientUtil;

    Logger logger = LoggerFactory.getLogger(LeJuCrawler.class);

    public JsonResult crawl() {

        String[] types = new String[]{"huodong", "shichang", "zhengce", "tudi", "yejie", "pingce", "daogou"};
        Document doc = webClientUtil.getDoc("http://bj.leju.com/cityguide/");
        Elements elements = doc.select("div#tbc_13>div.leter_list>ul>li>a");
        if (elements == null || elements.size() == 0) {
            return error("获取城市域名有误");
        }
        int success = 0;
        for (Element element : elements) {
            String domain = element.attr("href");
            //http://dl.leju.com/
            // String domain = type.substring(0,type.indexOf("."));
            doc = webClientUtil.getDoc(domain + "/news/");
            //如果新闻页内容为空说明没有新闻
            if (doc == null || doc.select("ul.new-list").size() == 0) {
                continue;
            }
            //http://dl.leju.com/news/huodong/
            int page = 1;
            for (String type : types) {
                for (int i = 0; true; i++) {
                    String link = domain + "/news/" + type + "/p-" + page + ".html";
                    if (type.equals("zhengce")) {
                        link = domain.substring(0, domain.indexOf(".")) + ".house.sina.com.cn/news/zhengce/p-" + page + ".html";
                    }
                    Document document = webClientUtil.getDoc(link);
                    if (document == null) {
                        break;
                    }
                    Elements articleElements = document.select("a.con-title");
                    if (articleElements == null || articleElements.size() == 0) {
                        break;
                    }
                    int ero = 0;
                    int warm = 0;
                    boolean end = false;
                    for (Element articleElement : articleElements) {
                        String src = articleElement.attr("href");
                        if (StringUtils.isBlank(src)) {
                            continue;
                        }
                        String text = webClientUtil.getDoc(src).select("div.moreinfo").text();
                        if (!text.contains("本公司保留追究法律责任的权利")) {
                            continue;
                        }
                        JsonResult jsonResult = articleCrawlerController.crawl(4313, src);
                        if (jsonResult.getStatus() == JsonResult.STATUS_ERROR) {
                            ero++;
                            if (ero == 15) {
                                return error("连续出错15条，核实原因");
                            }
                        } else if (jsonResult.getStatus() == JsonResult.STATUS_WARN) {
                            if (++warm >= 15) {
                                end = true;
                            }
                        } else {
                            ero = 0;
                            warm = 0;
                            success++;
                        }
                    }
                    //如果15条全是已经导入的文章，则该类别已经更新完毕，更新下一个类别
                    if (end) {
                        break;
                    }
                    page++;
                }
            }
        }
        return success("成功添加" + success + "条文章");
    }
}
